# -*- coding: UTF-8 -*-
# cython: language_level=3
from tqdm import tqdm

from db.mongodb import MongoDB
from db.oracle_db import OracleDB
from utils.own_tools import get_host

db = MongoDB()
ora_db = OracleDB()

with open(r"C:\Users\Administrator\Desktop\WEB_urls\WEB_urls.csv",encoding='utf-8') as f:
    x = f.readlines()
for url in tqdm(x):
    if not url or len(url) < 4 or '.' not in url:
        continue
    url = url.strip()

    if not url.startswith('http'):
        url = 'https://' + url
    # print(url)

    try:
        host = get_host(url)
        home_page = host['home_page']
        host = host['host']
        db.add('todo_urls', {'home_page': home_page, 'host': host, 'status': 0})
    except:
        pass
